package crawl

import (
	"encoding/csv"
	"fmt"
	"log"
	"os"

	"github.com/gocolly/colly/v2"
)

func CrawlTable() {
	c := colly.NewCollector(
	// Visit only domains: hackerspaces.org, wiki.hackerspaces.org
	// colly.AllowedDomains("s.askci.com"),
	)

	records := make([][]string, 0)
	// records = append(records, []string{"code", "company", "date"})

	c.OnHTML("table.fancyTable tbody", func(h *colly.HTMLElement) {
		fmt.Println("find it!")
		h.ForEach("tr", func(i int, e *colly.HTMLElement) {
			date := e.ChildText("td:nth-child(10)")
			code := e.ChildText("td:nth-child(2)")
			company := e.ChildText("td:nth-child(3)")
			records = append(records, []string{code, company, date})
		})
	})

	// Before making a request print "Visiting ..."
	c.OnRequest(func(r *colly.Request) {
		fmt.Println("Visiting", r.URL.String())
	})
	// todo
	for i := 101; i <= 150; i++ {
		c.Visit(fmt.Sprintf("http://api.scraperapi.com?api_key=70caeba64f903cf529ca6c1ee3fd0676&url=https://s.askci.com/stock/a/0-0?reportTime=2022-09-30&pageNum=%d", i))
	}
	// c.Visit("https://s.askci.com/stock/a/0-0?reportTime=2022-09-30&pageNum=248")

	Write2CSV(records, "time2market.csv")
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func Write2CSV(records [][]string, filename string) {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0666)
	check(err)

	defer f.Close()

	w := csv.NewWriter(f)
	for _, record := range records {
		if err := w.Write(record); err != nil {
			log.Fatalln("error writing record to csv:", err)
		}
	}

	w.Flush()
	if err := w.Error(); err != nil {
		log.Fatal(err)
	}
}
